# -*- coding: utf-8 -*-
__author__ = 'lztkdr'
__date__ = '2019/7/26 14:57'

from scrapy import signals

class HeadersMiddleware(object):

	@classmethod
	def from_crawler(cls, crawler):
		s = cls()
		crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
		return s

	def process_request(self, request, spider):
		if spider.mode == 1:
			# 请求头里 添加 Cookie
			request.headers.setdefault("Cookie", spider.antipas_cookie)
			#设置spider 中的 userAgent ，当更换代理时，同时也更换 userAgent
			request.headers.setdefault("user-agent", spider.useragent)
			#设置Referer
			request.headers.setdefault("Referer", spider.search_keyword_url)
		return None

	def process_response(self, request, response, spider):
		return response

	def process_exception(self, request, exception, spider):
		print(f"出现异常：{exception}")

	def spider_opened(self, spider):
		spider.logger.info('Spider opened: %s' % spider.name)